﻿(* filckr downloader *)
#r "FSharp.PowerPack.dll"

open Microsoft.FSharp.Control.WebExtensions
open System.IO
open System.Net

let fetchUrl (url:string) = 
    async {
        try 
            let req = WebRequest.Create(url) :?> HttpWebRequest
            req.UserAgent <- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)";
            req.Method <- "GET";
            req.AllowAutoRedirect <- true;
            req.MaximumAutomaticRedirections <- 4;
            let! response1 = req.AsyncGetResponse()
            let response = response1 :?> HttpWebResponse
            use stream = response.GetResponseStream()
            use streamreader = new System.IO.StreamReader(stream)
            return! streamreader.AsyncReadToEnd() // .ReadToEnd()
        with
            _ -> return "" // if there's any exception, just return an empty string
    }


let getImageUrls (page:string) = 
    let start = "<span class=\"photo_container pc_t\"><a href=\""
    let slen = start.Length
    let getUrl idx = 
        let idx2 = page.IndexOf('"', idx + slen)
        page.Substring(idx+slen, idx2-idx-slen)

    let rec scan (pos:int) = 
        seq {
            let idx = page.IndexOf(start, pos)
            if idx <> -1 then
                let url = getUrl idx
                yield ("http://www.flickr.com/" + url)
                yield! scan (idx+1)
        }
    scan 0

let getFileName (url:string) = 
    let s = url.Split '/'
    s.[s.Length-1]

let getImage (imageUrl:string) =
    async {
        try 
            let req = WebRequest.Create(imageUrl) :?> HttpWebRequest
            req.UserAgent <- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)";
            req.Method <- "GET";
            req.AllowAutoRedirect <- true;
            req.MaximumAutomaticRedirections <- 4;
            let! response1 = req.AsyncGetResponse()
            let response = response1 :?> HttpWebResponse
            use stream = response.GetResponseStream()
            let ms = new MemoryStream()
            let bytesRead = ref 1
            let buffer = Array.create 0x1000 0uy
            while !bytesRead > 0 do
                bytesRead := stream.Read(buffer, 0, buffer.Length)
                ms.Write(buffer, 0, !bytesRead)
            return ms.ToArray();

        with
            _ -> return Array.create 0 0uy // if there's any exception, just return an empty image
    }

let getBetween (page:string) (head:string) = 
    let len = head.Length
    let idx = page.IndexOf(head)
    let idx2 = page.IndexOf('"', idx+len)
    let between = page.Substring(idx+len, idx2 - idx - len)
    between


let getImageUrlAndTags (page:string) = 
    let header = "class=\"photoImgDiv\">"
    let idx = page.IndexOf(header)
    let url = getBetween (page.Substring(idx)) "<img src=\""

    let header2 = "<meta name=\"keywords\" content=\""
    let tagStr = getBetween page header2

    let s = tagStr.Split([|','|], System.StringSplitOptions.RemoveEmptyEntries)
    let tags = s |> Array.map (fun t -> t.Trim())
    url, tags

//(http://www.flickr.com/search/?q=dog&m=tags&s=int&page=2
let getImagesWithTag (tag:string) (pages:int) = 
    let rooturl = @"http://www.flickr.com/search/?q="+tag+"&m=tags&s=int"
    seq {
        for i=1 to pages do 
            let url = rooturl + "&page=" + i.ToString() 
            printfn "url = %s" url
            let page = fetchUrl url |> Async.RunSynchronously
            let imageUrls = getImageUrls page
            let getName (iurl:string) = 
                let s = iurl.Split '/'
                s.[s.Length-1]
            
            (* images in every search page *) 
            let images = 
                imageUrls 
                |> Seq.map (fun url -> fetchUrl url) 
                |> Async.Parallel
                |> Async.RunSynchronously
                |> Seq.map (fun page -> 
                    async {
                        let iurl, tags = getImageUrlAndTags page
                        let icontent = getImage iurl |> Async.RunSynchronously
                        let iname = getName iurl
                        return iname, icontent, tags
                    })
                |> Async.Parallel
                |> Async.RunSynchronously
            yield! images
        }


let downloadImagesWithTag (tag:string) (pages:int) (folder:string) = 
    let images = getImagesWithTag tag pages
    images
    |> Seq.iter (fun (name, content, tags) -> 
        let fname = folder + name
        File.WriteAllBytes(fname, content)
        File.WriteAllLines(fname + ".tag", tags)
        )




(* google result downloader *)
open System.Text.RegularExpressions

let getGoogleResultUrls (page:string) = 
    let getUrl idx = 
        let idx2 = page.IndexOf('"', idx)
        page.Substring(idx, idx2-idx)

    let rec urlPos (pos:int) = 
        let idx = page.IndexOf("<li class=", pos)
        
        if idx = -1 then 
            -1
        else
            //printfn "%s" (page.Substring(idx, 40))
            let idx2 = page.Substring(idx, 40).IndexOf("href=\"")
            let idxx = page.IndexOf("h3 class=", pos)
            if idx2 <> -1 && idxx <> -1 then // have both "h3 class" and "href"
                idx + idx2 + 6
            else
                urlPos (idx+1)

    let rec scan (pos:int) = 
        seq {
            let idx = urlPos pos
            if idx <> -1 then
                let url = getUrl idx
                if url.StartsWith("http://") then
                    yield url
                yield! scan (idx+1)
        }
    scan 0

let html2text (page:string) = 
    let rec removeScriptBlock (strData:string) = 
        let idx = strData.IndexOf("<script")
        if idx = -1 then
            strData
        else
            let idx2 = strData.IndexOf("</script", idx)
            if idx2 <> -1 then
                removeScriptBlock (strData.Remove(idx, idx2 - idx + 9))
            else
                strData
    let rec removeCommentBlock (strData:string) = 
        let idx = strData.IndexOf("<!--")
        if idx = -1 then
            strData
        else
            let idx2 = strData.IndexOf("--!", idx)
            if idx2 <> -1 then
                removeCommentBlock (strData.Remove(idx, idx2 - idx + 3))
            else
                strData
    let page2 = page.ToLower() |> removeScriptBlock // |> removeCommentBlock
    Regex.Replace(page2, "<(.|\n)*?>", "")

let downloadSearchPage (page:string) =
    let pages = 
        page 
        |> getGoogleResultUrls 
        |> Seq.map fetchUrl 
        |> Async.Parallel 
        |> Async.RunSynchronously
        |> Array.map html2text
    pages

let downloadGoogleQuery (query:string) (npages:int) (folder:string) = 
    for i=0 to npages-1 do
        let start = 10*i
        let g = @"http://www.google.com/search?q="+query+"&hl=en&prmd=invfd&ei=wrk5TJuDFo_ZcaSphPsO&start="+start.ToString()+"&sa=N"
        let page = fetchUrl g |> Async.RunSynchronously
        let pages = downloadSearchPage page
        pages 
        |> Array.iteri (fun j page ->
            File.WriteAllText(sprintf "%s%d_%d.txt" folder i j, page)
            )




downloadGoogleQuery "starfish" 30 @"D:\WORK\ImageData\googleResults\starfish\"

let a = File.ReadAllText(@"D:\WORK\ImageData\task\1\mytags.txt")
let s = a.Split ' '
s.Length
s.[3382]

(*
let dogs = downloadSearchPage page
let g = @"http://www.google.com/search?q=dog&hl=en&prmd=invfd&ei=w_YxTICqG42TkAXNuLWhDA&start=80&sa=N"
let page = fetchUrl g |> Async.RunSynchronously
//File.WriteAllText("d:\\g0.html", page)
let urls = getGoogleResultUrls page |> Seq.toList

*)




(*
let g = @"http://www.google.com/#q=dog&hl=en" //&prmd=invfd&ei=HvUxTMaaCsmrcbDp-OMD&start=20&sa=N&fp=4cb5f508e49085f6
let cake = @"http://www.flickr.com/search/?w=all&q=cake&m=tags"
let page = fetchUrl g |> Async.RunSynchronously
let urlsInPage = getImageUrls page |> Seq.toList
let imageUrl = @"http://farm2.static.flickr.com/1236/4725316406_02a6d3fe78.jpg"
let i = getImage imageUrl |> Async.RunSynchronously
let writeImage imageArr path = 
    File.WriteAllBytes(path, imageArr)


writeImage i "d:\\temp.jpg"
let pp = @"http://www.flickr.com/photos/stellacake/4724610493/" |> fetchUrl |> Async.RunSynchronously
let pp2 = @"http://www.flickr.com/photos/serenae/4724392514/" |> fetchUrl |> Async.RunSynchronously
getImageUrlAndTags pp
getImageUrlAndTags pp2
*)

downloadImagesWithTag "sheep" 5 @"D:\WORK\ImageData\flickr\sheep\"




(*
let url = @"http://www.flickr.com/search/?w=all&q=cat&m=tags#page=2"
let page = fetchUrl url |> Async.RunSynchronously
File.WriteAllText(@"d:\a.html", page)
*)

 


